#!/bin/bash
#############################################################################
# Description:告警管理脚本                                                   #                                               
# date: 2023-05-17                                                          #                                                                           
# Emain: 910354217@qq.com                                                   #                                                         
# Explanation： 1.根据不同项目,告到不同钉钉群里 2.优化频繁告警现象,渐变式告警    #                            
# For example：*/1 * * * *  /bin/bash /app/shell/alarm_management.sh        #                                                           
#############################################################################
#环境变量
basepas=/app/shell
sendpwd=$basepas/libs
logpwd=$basepas/log

#原始临时文件
source_tmp_file=$logpwd/source_tmp.log
source_tmptime_file=$logpwd/source_tmptime.log

#原始告警文件
sourcefile=$logpwd/test_source.log
#过滤表
filter_file=$logpwd/filter.log
#过滤以后的告警文件
result_file=$logpwd/result_file.log
#md5值判断文件
md5_file=$logpwd/md5_value.log
#发出告警的信息文件
alarm_file=$logpwd/alarm_file.log
alarm_status_file=$logpwd/alarm_file_status.log

#钉钉相关信息文件
dingdingcsv=$basepas/data/dingding.csv

#将promethous告警值get下来
get_alarm(){
    #curl $promethos_alarm_url |  jq '.data.alerts[] | {a1:.labels.alertname,a2:.labels.bindip,a3:.labels.project,a4:.labels.quota,a5:.labels.servicename,a6:.labels.severity,a7:.labels.value,a8:.activeAt}' |  jq -r '[.[]] | @csv' |   sed 's/\"//g'  | sort -t ','  -k1  -k2  |  grep -v PROCESS>$sourcefile
    #获取告警信息地址
    promethos_alarm_url="http://192.168.2.200:60002/api/v1/alerts"
    curl $promethos_alarm_url |  jq '.data.alerts[].labels' | jq -r '[.[]] | @csv'> $source_tmp_file
    curl $promethos_alarm_url |  jq '.data.alerts[].activeAt'> $source_tmptime_file

    paste -d ',' $source_tmp_file  $source_tmptime_file |   sed 's/\"//g'   |  grep -v PROCESS >$sourcefile
}


#过滤不需要的告警信息
filter(){
    #初始化数据
    >$result_file  
    #将过滤项追加到过滤后告警文件
    for filterValue in `cat $filter_file | grep -v '^#' | sort | uniq`
    do
        #根据告警名称和告警ip去进行对应过滤
        filterName=`echo $filterValue| awk -F ',' '{print $1}'`
        filerHost=`echo $filterValue| awk -F ',' '{print $2}'`
        cat $sourcefile | awk -v filterName=$filterName -F ',' '{ if  ($1 == filterName) print $0}' |  grep -Evi "$filerHost" >>$result_file
        #将过滤项变成|号追加
        let index+=1
        if [[ $index -eq 1 ]];then
            loseName+="$filterName"
        else
            loseName+="|$filterName"
        fi
    done
    #将非过滤项追加到过滤后告警文件
    if [[ $loseName == "" ]];then
        echo ""
    else
        cat $sourcefile | grep -Ev "$loseName" >>$result_file
    fi
}

#格式化告警信息,并进行钉钉告警
format_alarmfile(){
    #获取主题,ip,项目名,quota,服务名,等级,值,时间
    for alarm_line in `cat $alarm_file`   
    do
        theme=`echo $alarm_line | awk -F ',' '{print $1}'`
        bindip=`echo $alarm_line | awk -F ',' '{print $2}'`
        project=`echo $alarm_line | awk -F ',' '{print $6}'`
        quota=`echo $alarm_line | awk -F ',' '{print $7}'`
        servername=`echo $alarm_line | awk -F ',' '{print $8}'`
        severity=`echo $alarm_line | awk -F ',' '{print $9}'`
        value=`echo $alarm_line | awk -F ',' '{print $10}'`
        datetime=`echo $alarm_line |  awk -F ',' '{print $11}' | awk -F '.' '{print $1}'`   
        #获取钉钉相关参数
        dingding_url=`grep $project $dingdingcsv |  awk -F ',' '{print $2}'`
        if [[ $dingding_url == ""   ]];then
            dingding_url="https://oapi.dingtalk.com/robot/send?access_token=02fd5264b64d3907ec768ad1b133f739cb5720e7e90148d4e6ca37cf34a83a94"
            dingding_iphone="18820304608" 
        else
            dingding_iphone=`grep $project $dingdingcsv |  awk -F ',' '{print $3}'`
        fi
        
        echo $status $quota   $theme $project $servername  $bindip $severity $value $datetime  $dingding_url $dingding_iphone
        #因为发生钉钉时间比较长,进行并发告警
        python3 $sendpwd/send_prometheus.py  $status  $theme $project $servername  $bindip $severity $value $datetime  $dingding_url $dingding_iphone 
    done  
}


#智能告警管理



#告警管理
majorAlarm(){
    #先判断是否存在恢复告警
    for value in `cat $result_file`
    do
       md5_value=`echo $value | awk -F, 'OFS=",",NF-=1' | md5sum | awk '{print $1}'`
       status="恢复告警"
       grep  $md5_value $alarm_file ||  format_alarmfile  
    done   

    #在判断是否有告警信息
    if  [[  `cat $result_file | wc -l` == 0   ]];then
        echo "告警没变化"
    else
        echo "存在新增告警信息"
        #给每个监控项一个md5值
        index_num=0
        >$alarm_file
        #>alarm_status_file
        for value in `cat $result_file`
        do
            md5_value=`echo $value | awk -F, 'OFS=",",NF-=1' | md5sum | awk '{print $1}'`
            echo $value,$md5_value >>$alarm_file
            grep $md5_value  $alarm_status_file || echo 1 
        done
        status="新增告警"
        format_alarmfile
    fi
}


get_alarm
filter
majorAlarm


# 主题:cpu使用率大于80%
# 项目:中翰科技
# 角色:db
# ip:8.134.23.226
# 等级:严重
# 值:92
# 告警发生时间:2023-05-16-11:50
# 持续时间:
# 状态:

